package crawler.web.other;

import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;

import org.htmlparser.Node;
import org.htmlparser.PrototypicalNodeFactory;
import org.htmlparser.lexer.Lexer;
import org.htmlparser.util.ParserException;

/**
 * @author waynechen Feb 14, 2012 10:52:55 AM
 */
public class PageSimilarity {

	public static void main(String[] args) throws ParserException, IOException {
		String urlStr1 = "http://news.baidu.com/n?cmd=1&class=civilnews&pn=1&from=tab";
		String urlStr2 = "http://news.baidu.com/n?cmd=1&class=internews&pn=1&from=tab";
		String urlStr3 = "http://ehr.chinahr.com/Login.aspx?ReturnUrl=%2fResume%2fResumeCenter%2fPerchaseResumeListNew.aspx%3ftype%3d0%26navigator%3dNavNode_3&type=0&navigator=NavNode_3";
		System.out.println("网页相似度:" + getPageSim(urlStr1, urlStr2));
		//-------
		System.out.println("网页相似度:" + getPageSim(urlStr1, urlStr3));
	}

	public static double getPageSim(String urlStr1, String urlStr2) throws ParserException, IOException {
		Node node;
		Lexer lexer;
		ArrayList<Node> pageNodes1 = new ArrayList<Node>();
		ArrayList<Node> pageNodes2 = new ArrayList<Node>();
		URL url1 = new URL(urlStr1);
		URL url2 = new URL(urlStr2);

		lexer = new Lexer(url1.openConnection());
		lexer.setNodeFactory(new PrototypicalNodeFactory());
		while (null != (node = lexer.nextNode())) {
			pageNodes1.add(node);
		}

		lexer = new Lexer(url2.openConnection());
		lexer.setNodeFactory(new PrototypicalNodeFactory());
		while (null != (node = lexer.nextNode())) {
			pageNodes2.add(node);
		}

		System.out.println("页面距离:" + PageDistance.LD(pageNodes1, pageNodes2));

		int distance = Subsequence.LCSLength(pageNodes1.toArray(), pageNodes2.toArray());
		return (2.0 * distance) / (pageNodes1.size() + pageNodes2.size());
	}

}
